{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|default_exp models.ConvTranPlus"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ConvTranPlus"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ">ConvTran: Improving Position Encoding of Transformers for Multivariate Time Series Classification\n",
    "\n",
    "This is a Pytorch implementation of ConvTran adapted by Ignacio Oguiza and based on:\n",
    "\n",
    "Foumani, N. M., Tan, C. W., Webb, G. I., & Salehi, M. (2023). Improving Position Encoding of Transformers for Multivariate Time Series Classification. arXiv preprint arXiv:2305.16642.\n",
    "\n",
    "Pre-print: https://arxiv.org/abs/2305.16642v1\n",
    "\n",
    "Original repository:  https://github.com/Navidfoumani/ConvTran"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "from collections import OrderedDict\n",
    "from typing import Any\n",
    "\n",
    "import math\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from tsai.models.layers import lin_nd_head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "class tAPE(nn.Module):\n",
    "    \"time Absolute Position Encoding\"\n",
    "\n",
    "    def __init__(self, \n",
    "        d_model:int, # the embedding dimension\n",
    "        seq_len=1024, # the max. length of the incoming sequence\n",
    "        dropout:float=0.1, # dropout value\n",
    "        scale_factor=1.0\n",
    "        ):\n",
    "        super().__init__()\n",
    "        \n",
    "        pe = torch.zeros(seq_len, d_model)  # positional encoding\n",
    "        position = torch.arange(0, seq_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "\n",
    "        pe[:, 0::2] = torch.sin((position * div_term)*(d_model/seq_len))\n",
    "        pe[:, 1::2] = torch.cos((position * div_term)*(d_model/seq_len))\n",
    "        pe = scale_factor * pe.unsqueeze(0)\n",
    "        self.register_buffer('pe', pe)  # this stores the variable in the state_dict (used for non-trainable variables)\n",
    "\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "    def forward(self, x): # [batch size, sequence length, embed dim]\n",
    "        x = x + self.pe\n",
    "        return self.dropout(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = torch.randn(8, 50, 128)\n",
    "assert tAPE(128, 50)(t).shape == t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "class AbsolutePositionalEncoding(nn.Module):\n",
    "    \"Absolute positional encoding\"\n",
    "\n",
    "    def __init__(self, \n",
    "        d_model:int, # the embedding dimension\n",
    "        seq_len=1024, # the max. length of the incoming sequence\n",
    "        dropout:float=0.1, # dropout value\n",
    "        scale_factor=1.0\n",
    "        ):\n",
    "        super().__init__()\n",
    "\n",
    "        \n",
    "        pe = torch.zeros(seq_len, d_model)  # positional encoding\n",
    "        position = torch.arange(0, seq_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = scale_factor * pe.unsqueeze(0)\n",
    "        self.register_buffer('pe', pe)  # this stores the variable in the state_dict (used for non-trainable variables)\n",
    "\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "    def forward(self, x): # [batch size, sequence length, embed dim]\n",
    "        x = x + self.pe\n",
    "        return self.dropout(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = torch.randn(8, 50, 128)\n",
    "assert AbsolutePositionalEncoding(128, 50)(t).shape == t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "class LearnablePositionalEncoding(nn.Module):\n",
    "    \"Learnable positional encoding\"\n",
    "\n",
    "\n",
    "    def __init__(self, \n",
    "        d_model:int, # the embedding dimension\n",
    "        seq_len=1024, # the max. length of the incoming sequence\n",
    "        dropout:float=0.1, # dropout value\n",
    "        ):\n",
    "        super().__init__()\n",
    "\n",
    "        self.pe = nn.Parameter(torch.empty(seq_len, d_model))  # requires_grad automatically set to True\n",
    "        nn.init.uniform_(self.pe, -0.02, 0.02)\n",
    "\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "\n",
    "    def forward(self, x): #[batch size, seq_len, embed dim]\n",
    "        x = x + self.pe\n",
    "        return self.dropout(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = torch.randn(8, 50, 128)\n",
    "assert LearnablePositionalEncoding(128, 50)(t).shape == t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "class Attention(nn.Module):\n",
    "    def __init__(self, \n",
    "        d_model:int, # Embedding dimension\n",
    "        n_heads:int=8, # number of attention heads\n",
    "        dropout:float=0.01, # dropout\n",
    "        ):\n",
    "        super().__init__()\n",
    "        self.n_heads = n_heads\n",
    "        self.scale = d_model ** -0.5\n",
    "        self.key = nn.Linear(d_model, d_model, bias=False)\n",
    "        self.value = nn.Linear(d_model, d_model, bias=False)\n",
    "        self.query = nn.Linear(d_model, d_model, bias=False)\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.to_out = nn.LayerNorm(d_model)\n",
    "\n",
    "    def forward(self, x): #[batch size, seq_len, embed dim]\n",
    "\n",
    "        batch_size, seq_len, _ = x.shape\n",
    "        k = self.key(x).reshape(batch_size, seq_len, self.n_heads, -1).permute(0, 2, 3, 1)\n",
    "        v = self.value(x).reshape(batch_size, seq_len, self.n_heads, -1).transpose(1, 2)\n",
    "        q = self.query(x).reshape(batch_size, seq_len, self.n_heads, -1).transpose(1, 2)\n",
    "\n",
    "        attn = torch.matmul(q, k) * self.scale\n",
    "        attn = nn.functional.softmax(attn, dim=-1)\n",
    "\n",
    "        out = torch.matmul(attn, v) # [batch_size, n_heads, seq_len, d_head]\n",
    "        out = out.transpose(1, 2) # [batch_size, seq_len, n_heads, d_head]\n",
    "        out = out.reshape(batch_size, seq_len, -1) # [batch_size, seq_len, d_model]\n",
    "        out = self.to_out(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = torch.randn(8, 50, 128)\n",
    "assert Attention(128)(t).shape == t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "class Attention_Rel_Scl(nn.Module):\n",
    "    def __init__(self, \n",
    "        d_model:int, # Embedding dimension\n",
    "        seq_len:int, # sequence length\n",
    "        n_heads:int=8, # number of attention heads\n",
    "        dropout:float=0.01, # dropout\n",
    "        ):\n",
    "        super().__init__()\n",
    "\n",
    "        self.seq_len = seq_len\n",
    "        self.n_heads = n_heads\n",
    "        self.scale = d_model ** -0.5\n",
    "\n",
    "        self.key = nn.Linear(d_model, d_model, bias=False)\n",
    "        self.value = nn.Linear(d_model, d_model, bias=False)\n",
    "        self.query = nn.Linear(d_model, d_model, bias=False)\n",
    "\n",
    "        self.relative_bias_table = nn.Parameter(torch.zeros((2 * self.seq_len - 1), n_heads))\n",
    "        coords = torch.meshgrid((torch.arange(1), torch.arange(self.seq_len)), indexing=\"xy\")\n",
    "        coords = torch.flatten(torch.stack(coords), 1)\n",
    "        relative_coords = coords[:, :, None] - coords[:, None, :]\n",
    "        relative_coords[1] += self.seq_len - 1\n",
    "        relative_coords = relative_coords.permute(1, 2, 0)\n",
    "        relative_index = relative_coords.sum(-1).flatten().unsqueeze(1)\n",
    "        self.register_buffer(\"relative_index\", relative_index)\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.to_out = nn.LayerNorm(d_model)\n",
    "\n",
    "    def forward(self, x): #[batch size, seq_len, embed dim]\n",
    "        batch_size, seq_len, _ = x.shape\n",
    "        k = self.key(x).reshape(batch_size, seq_len, self.n_heads, -1).permute(0, 2, 3, 1)\n",
    "        v = self.value(x).reshape(batch_size, seq_len, self.n_heads, -1).transpose(1, 2)\n",
    "        q = self.query(x).reshape(batch_size, seq_len, self.n_heads, -1).transpose(1, 2)\n",
    "\n",
    "        attn = torch.matmul(q, k) * self.scale # [seq_len, seq_len]\n",
    "        attn = nn.functional.softmax(attn, dim=-1)\n",
    "\n",
    "        relative_bias = self.relative_bias_table.gather(0, self.relative_index.repeat(1, 8))\n",
    "        relative_bias = relative_bias.reshape(self.seq_len, self.seq_len, -1).permute(2, 0, 1).unsqueeze(0)\n",
    "        attn = attn + relative_bias\n",
    "\n",
    "        out = torch.matmul(attn, v) # [batch_size, n_heads, seq_len, d_head]\n",
    "        out = out.transpose(1, 2) # [batch_size, seq_len, n_heads, d_head]\n",
    "        out = out.reshape(batch_size, seq_len, -1) # [batch_size, seq_len, d_model]\n",
    "        out = self.to_out(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = torch.randn(8, 50, 128)\n",
    "assert Attention_Rel_Scl(128, 50)(t).shape == t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "class Attention_Rel_Vec(nn.Module):\n",
    "    def __init__(self, \n",
    "        d_model:int, # Embedding dimension\n",
    "        seq_len:int, # sequence length\n",
    "        n_heads:int=8, # number of attention heads\n",
    "        dropout:float=0.01, # dropout\n",
    "        ):\n",
    "        super().__init__()\n",
    "\n",
    "        self.seq_len = seq_len\n",
    "        self.n_heads = n_heads\n",
    "        self.scale = d_model ** -0.5\n",
    "\n",
    "        self.key = nn.Linear(d_model, d_model, bias=False)\n",
    "        self.value = nn.Linear(d_model, d_model, bias=False)\n",
    "        self.query = nn.Linear(d_model, d_model, bias=False)\n",
    "\n",
    "        self.Er = nn.Parameter(torch.randn(self.seq_len, int(d_model/n_heads)))\n",
    "\n",
    "        self.register_buffer(\n",
    "            \"mask\",\n",
    "            torch.tril(torch.ones(self.seq_len, self.seq_len))\n",
    "            .unsqueeze(0).unsqueeze(0)\n",
    "        )\n",
    "\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.to_out = nn.LayerNorm(d_model)\n",
    "\n",
    "    def forward(self, x): #[batch size, seq_len, embed dim]\n",
    "        batch_size, seq_len, _ = x.shape\n",
    "        k = self.key(x).reshape(batch_size, seq_len, self.n_heads, -1).permute(0, 2, 3, 1) # [batch_size, n_heads, seq_len, d_head]\n",
    "        v = self.value(x).reshape(batch_size, seq_len, self.n_heads, -1).transpose(1, 2) # [batch_size, n_heads, seq_len, d_head]\n",
    "        q = self.query(x).reshape(batch_size, seq_len, self.n_heads, -1).transpose(1, 2) # [batch_size, n_heads, seq_len, d_head]\n",
    "\n",
    "        QEr = torch.matmul(q, self.Er.transpose(0, 1))\n",
    "        Srel = self.skew(QEr) # [batch_size, self.n_heads, seq_len, seq_len]\n",
    "\n",
    "        attn = torch.matmul(q, k) # [seq_len, seq_len]\n",
    "        attn = (attn + Srel) * self.scale\n",
    "\n",
    "        attn = nn.functional.softmax(attn, dim=-1)\n",
    "        out = torch.matmul(attn, v) # [batch_size, n_heads, seq_len, d_head]\n",
    "        out = out.transpose(1, 2) # [batch_size, seq_len, n_heads, d_head]\n",
    "        out = out.reshape(batch_size, seq_len, -1) # [batch_size, seq_len, d_model]\n",
    "        out = self.to_out(out)\n",
    "        return out\n",
    "\n",
    "    def skew(self, QEr): # [batch_size, n_heads, seq_len, seq_len]\n",
    "        padded = nn.functional.pad(QEr, (1, 0)) # [batch_size, n_heads, seq_len, 1 + seq_len]\n",
    "        batch_size, n_heads, num_rows, num_cols = padded.shape\n",
    "        reshaped = padded.reshape(batch_size, n_heads, num_cols, num_rows) # [batch_size, n_heads, 1 + seq_len, seq_len]\n",
    "        Srel = reshaped[:, :, 1:, :] # [batch_size, n_heads, seq_len, seq_len]\n",
    "        return Srel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = torch.randn(8, 50, 128)\n",
    "assert Attention_Rel_Vec(128, 50)(t).shape == t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "class ConvTranBackbone(nn.Module):\n",
    "    def __init__(self, \n",
    "        c_in:int, \n",
    "        seq_len:int, \n",
    "        d_model=16, # Internal dimension of transformer embeddings\n",
    "        n_heads:int=8, # Number of multi-headed attention heads\n",
    "        dim_ff:int=256, # Dimension of dense feedforward part of transformer layer\n",
    "        abs_pos_encode:str='tAPE', # Absolute Position Embedding. choices={'tAPE', 'sin', 'learned', None}\n",
    "        rel_pos_encode:str='eRPE', # Relative Position Embedding. choices={'eRPE', 'vector', None}\n",
    "        dropout:float=0.01, # Droupout regularization ratio\n",
    "        ):\n",
    "        super().__init__()\n",
    "\n",
    "\n",
    "        self.embed_layer = nn.Sequential(nn.Conv2d(1, d_model*4, kernel_size=[1, 7], padding='same'), nn.BatchNorm2d(d_model*4), nn.GELU())\n",
    "        self.embed_layer2 = nn.Sequential(nn.Conv2d(d_model*4, d_model, kernel_size=[c_in, 1], padding='valid'), nn.BatchNorm2d(d_model), nn.GELU())\n",
    "\n",
    "        assert abs_pos_encode in ['tAPE', 'sin', 'learned', None]\n",
    "        if abs_pos_encode == 'tAPE':\n",
    "            self.abs_position = tAPE(d_model, dropout=dropout, seq_len=seq_len)\n",
    "        elif abs_pos_encode == 'sin':\n",
    "            self.abs_position = AbsolutePositionalEncoding(d_model, dropout=dropout, seq_len=seq_len)\n",
    "        elif abs_pos_encode== 'learned':\n",
    "            self.abs_position = LearnablePositionalEncoding(d_model, dropout=dropout, seq_len=seq_len)\n",
    "        else:\n",
    "            self.abs_position = nn.Identity()\n",
    "\n",
    "        assert rel_pos_encode in ['eRPE', 'vector', None]\n",
    "        if rel_pos_encode == 'eRPE':\n",
    "            self.attention_layer = Attention_Rel_Scl(d_model, seq_len, n_heads=n_heads, dropout=dropout)\n",
    "        elif rel_pos_encode == 'vector':\n",
    "            self.attention_layer = Attention_Rel_Vec(d_model, seq_len, n_heads=n_heads, dropout=dropout)\n",
    "        else:\n",
    "            self.attention_layer = Attention(d_model, n_heads=n_heads, dropout=dropout)\n",
    "\n",
    "        self.LayerNorm = nn.LayerNorm(d_model, eps=1e-5)\n",
    "        self.LayerNorm2 = nn.LayerNorm(d_model, eps=1e-5)\n",
    "\n",
    "        self.FeedForward = nn.Sequential(\n",
    "            nn.Linear(d_model, dim_ff),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(dropout),\n",
    "            nn.Linear(dim_ff, d_model),\n",
    "            nn.Dropout(dropout))\n",
    "\n",
    "    def forward(self, x): # [batch size, c_in, seq_len]\n",
    "        x = x.unsqueeze(1)\n",
    "        x_src = self.embed_layer(x)\n",
    "        x_src = self.embed_layer2(x_src).squeeze(2)\n",
    "        x_src = x_src.permute(0, 2, 1)\n",
    "        x_src_pos = self.abs_position(x_src)\n",
    "        att = x_src + self.attention_layer(x_src_pos)\n",
    "        att = self.LayerNorm(att)\n",
    "        out = att + self.FeedForward(att)\n",
    "        out = self.LayerNorm2(out)\n",
    "        out = out.permute(0, 2, 1)\n",
    "        return out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = torch.randn(8, 5, 20)\n",
    "assert ConvTranBackbone(5, 20)(t).shape, (8, 16, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "class ConvTranPlus(nn.Sequential):\n",
    "    def __init__(self, \n",
    "        c_in:int, # Number of channels in input\n",
    "        c_out:int, # Number of channels in output\n",
    "        seq_len:int, # Number of input sequence length\n",
    "        d:tuple=None,  # output shape (excluding batch dimension).\n",
    "        d_model:int=16, # Internal dimension of transformer embeddings\n",
    "        n_heads:int=8, # Number of multi-headed attention heads\n",
    "        dim_ff:int=256, # Dimension of dense feedforward part of transformer layer\n",
    "        abs_pos_encode:str='tAPE', # Absolute Position Embedding. choices={'tAPE', 'sin', 'learned', None}\n",
    "        rel_pos_encode:str='eRPE', # Relative Position Embedding. choices={'eRPE', 'vector', None}\n",
    "        encoder_dropout:float=0.01, # Droupout regularization ratio for the encoder\n",
    "        fc_dropout:float=0.1, # Droupout regularization ratio for the head\n",
    "        use_bn:bool=True, # indicates if batchnorm will be applied to the model head.\n",
    "        flatten:bool=True, # this will flatten the output of the encoder before applying the head if True.\n",
    "        custom_head:Any=None, # custom head that will be applied to the model head (optional).\n",
    "        ):\n",
    "        \"\"\n",
    "\n",
    "        # Backbone\n",
    "        backbone = ConvTranBackbone(c_in, seq_len, d_model=d_model, n_heads=n_heads, dim_ff=dim_ff, \n",
    "                                    abs_pos_encode=abs_pos_encode, rel_pos_encode=rel_pos_encode, dropout=encoder_dropout)\n",
    "\n",
    "        # Head\n",
    "        self.head_nf = d_model\n",
    "        if custom_head is not None: \n",
    "            if isinstance(custom_head, nn.Module): head = custom_head\n",
    "            else: head = custom_head(self.head_nf, c_out, seq_len)\n",
    "        elif d is not None:\n",
    "            head = lin_nd_head(self.head_nf, c_out, seq_len=seq_len, d=d, use_bn=use_bn, fc_dropout=fc_dropout, flatten=flatten)\n",
    "        else:\n",
    "            head = nn.Sequential(nn.AdaptiveAvgPool1d(1), nn.Flatten(), nn.Linear(self.head_nf, c_out))\n",
    "\n",
    "        layers = OrderedDict([('backbone', backbone), ('head', head)])\n",
    "        super().__init__(layers) \n",
    "\n",
    "ConvTran = ConvTranPlus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xb = torch.randn(16, 5, 20)\n",
    "\n",
    "model = ConvTranPlus(5, 3, 20, d=None)\n",
    "output = model(xb)\n",
    "assert output.shape == (16, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xb = torch.randn(16, 5, 20)\n",
    "\n",
    "model = ConvTranPlus(5, 3, 20, d=5)\n",
    "output = model(xb)\n",
    "assert output.shape == (16, 5, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xb = torch.randn(16, 5, 20)\n",
    "\n",
    "model = ConvTranPlus(5, 3, 20, d=(2, 10))\n",
    "output = model(xb)\n",
    "assert output.shape == (16, 2, 10, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": "IPython.notebook.save_checkpoint();",
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/nacho/notebooks/tsai/nbs/081_models.ConvTranPlus.ipynb saved at 2023-07-04 18:01:17\n",
      "Correct notebook to script conversion! 😃\n",
      "Tuesday 04/07/23 18:01:19 CEST\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <audio  controls=\"controls\" autoplay=\"autoplay\">\n",
       "                    <source src=\"data:audio/wav;base64,UklGRvQHAABXQVZFZm10IBAAAAABAAEAECcAACBOAAACABAAZGF0YdAHAAAAAPF/iPh/gOoOon6w6ayCoR2ZeyfbjobxK+F2Hs0XjKc5i3DGvzaTlEaraE+zz5uLUl9f46fHpWJdxVSrnfmw8mYEScqUP70cb0Q8X41uysJ1si6Eh1jYzXp9IE2DzOYsftYRyoCY9dJ/8QICgIcEun8D9PmAaBPlfT7lq4MFIlh61tYPiCswIHX+yBaOqT1QbuW7qpVQSv9lu6+xnvRVSlyopAypbGBTUdSalrSTaUBFYpInwUpxOzhti5TOdndyKhCGrdwAfBUcXIJB69p+Vw1egB76+n9q/h6ADglbf4LvnIHfF/981ODThF4m8HiS0riJVjQ6c+/EOZCYQfJrGrhBmPVNMmNArLKhQlkXWYqhbaxXY8ZNHphLuBJsZUEckCTFVHMgNKGJytIDeSUmw4QN4Qx9pReTgb3vYX/TCBuApf75f+P5Y4CRDdN+B+tngk8c8nt03CKGqipgd13OhotwOC5x9MCAknFFcmlmtPmagFFFYOCo0qRzXMhVi57pryNmIEqJlRi8bm52PfuNM8k4dfQv+4cO12l6zCGdg3jl730uE/KAPvS+f0wEAoAsA89/XfXQgBESIn6S5luDtiC8eh/YmIfpLqt1OMp5jXg8/24MveqUNUnPZsqw0Z3yVDldnaUOqIZfXlKrm36zzWhjRhaT+r+ncHI5/otUzfd2uSt7hl/bqXtoHaCC6+mqfrAOeoDD+PJ/xf8RgLMHfH/b8GeBihZIfSXidoQSJWB52NM1iRkzz3MkxpKPbUCrbDu5d5fgTAxkSK3JoEhYD1p2omere2LZTuqYLbdWa49Cx5Dww7tyXDUnioXRkHhwJyKFvd/AfPoYy4Fl7j1/LQorgEr9/X89+0qAOAwAf13sJoL8Gkd8wt25hWIp3Heez/eKODfPcSPCzpFNRDVqf7UlmnNQKGHgqd+jgVvJVm2f265QZTpLS5byur1tpT6ajvrHq3Q2MXWIxtUCehoj8YMk5LB9hRQegeTypn+nBQWA0QHgf7f2q4C5EFt+5ucOg2YfHXtq2SSHpS0ydnTL4IxFO6pvNb4ulBdInWfcsfSc7VMmXpSmE6eeXmZThJxpsgRohEfOk86+AHCoOpOMFsx1dv8s6oYT2k17uR7ngpXod34IEJqAaPfnfyABCIBZBpl/NPI2gTQVjX134x2ExSPMeR7VtYjZMWJ0W8ftjkA/YW1durCWykvjZFKu4p9LVwVbZKNkqpxh6U+6mRC2mGq2Q3SRvsIgcpc2sIpD0Bp4uiiFhW3ecXxOGgaCDe0Vf4cLPoDv+/5/mfw1gN4KKX+17emBqBmYfBHfVYUZKFR44NBtiv41bHJUwx+RJkP1apu2VJlkTwli4qrwoo1ax1dToNCtemRSTBGXz7kJbdM/PY/Dxht0dTLziH7Ul3loJEiE0uJsfdsVTYGL8Yt/AgcMgHYA7X8S+IqAYA+QfjzpxIIVHnp7tdqzhmAstXaxzEqMETpScGC/dJP3Rmdo8LIZnOVSEF+Opxumsl1sVF+dVrE5Z6NIiZSkvVdv2zsqjdnK8HVDLlyHyNjuegogM4NA5z9+YRG9gA722H97AgOA/gSyf43zCIHdE899yuTIg3ciNXpm1jmImTDwdJPITI4RPhRugbvslbFKt2Vfr/6eTFb4W1WkY6m6YPdQjJr2tNZp3EQlko7BgXHRNz2LAc+gdwMq7IUf3R58ohtFgrbr6n7hDFWAlPr8f/T9I4CECU9/De+vgVQY5nxh4POEzybJeCTS5YnCNAZzhsRzkP1Bsmu4t4aYU07nYuerA6KWWcJYO6HHrKJjaE3Zl624UWz/QOOPjcWHc7QzdIk40yl5tCWjhIDhJX0xF4CBMvBsf10IF4Ac//Z/bPlsgAcOwn6S6n6CwxzUewLcRoYaKzV38M23i9o493CNwL6S1UUuaQe0QpvbUfdfiqglpcRccFU+nkWwambASUiVfLyqbg49xY2eyWh1hy/Sh37XjHpaIYKD7OUEfrgS5IC09MV/1gMBgKMDyH/n9N6AhhINfh7mdoMoIZt6r9fAh1cvfHXNya6N4DzDbqi8K5WWSYlmbbAdnkpV6FxJpWSo1V8DUmGb3rMRaQBG2JJgwN9wCDnNi8HNI3dKK1aG0dvHe/UciIJf6rt+Og5wgDn59X9P/xWAKQhxf2XweYH+FjB9suGVhIMlOnlo02GJhTOdc7vFyo/TQGxs2Li7lz9NwmPurBihnVi7WSWiwKvGYntOpJiOt5drKUKMkFnE8HLxNPmJ9NG4eP8mAYUv4Np8hhi3gdruSX+3CSWAwP38f8f6UoCuDPF+6Os8gnAbKnxQ3d2F0imydzDPKIuiN5lxu8EKkrFE82kftW2az1DbYImpMqTUW3FWIJ83r5hl2koJlla7+m0+PmSOZcjcdMgwS4g11iZ6qCLUg5jkxn0QFA6BWvOvfzEFBIBHAtp/Qfa3gC4RSH5y5yeD2B/8evnYS4cULgR2CMsUja47cG/QvW6UeEhXZ3+xP51GVNVdP6Zpp+1eDFM5nMeySWghR4+TNL85cD46YIyCzKJ2kCzEhoTabXtGHs+CCemJfpMPjoDe9+t/qQALgM8Gj3++8UaBqRV2fQTjO4Q3JKd5r9TgiEYyMHTxxiWPpz8jbfq585YpTJpk960xoKFXsVoTo7yq6GGMTw==\" type=\"audio/wav\" />\n",
       "                    Your browser does not support the audio element.\n",
       "                </audio>\n",
       "              "
      ],
      "text/plain": [
       "<IPython.lib.display.Audio object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#|eval: false\n",
    "#|hide\n",
    "from tsai.export import get_nb_name; nb_name = get_nb_name(locals())\n",
    "from tsai.imports import create_scripts; create_scripts(nb_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
